
package org.sopt.dev.ImgKnight.agent.RobotsExclusion;

import java.net.URL;
import java.net.URLConnection;
import java.io.PushbackInputStream;
import java.io.BufferedInputStream;
import java.util.Vector;

public class RobotsExclusion {
	
	public static final int MAX_ENTRY_LENGTH = 1024;
	
	String UserAgent = null;
	
	RobotsTxtCacheEngine<String, String[]> cacheEngine = null;
	
	public RobotsExclusion(String UserAgent) {
		this(UserAgent, new DefaultCacheEngine(MAX_ENTRY_LENGTH));
	}
	
	public RobotsExclusion(String UserAgent, 
			RobotsTxtCacheEngine<String, String[]> cacheEngine) {
		this.UserAgent 		= UserAgent;
		this.cacheEngine 	= cacheEngine;
	}
	
	public void close() {
		cacheEngine.clear();
	}
	
	public boolean disallowd(URL url) {
		if( !url.getProtocol().startsWith("http") )
			return false;
		
		String website = getWebSite(url);
		String[] rules = cacheEngine.get(website);
		if( rules == null ) {
			rules = getRobotExclusionContext(website, UserAgent);
			cacheEngine.put(website, rules);
		}
		
		String path = url.getFile();
		for(int i=0; i<rules.length; ++i) {
			if( rules[i].equals("") )
				return false;
			if( path.startsWith(rules[i]) )
				return true;
		}
		return false;
	}
	
	String getWebSite(URL url) {
		String hostname = url.getHost();
		int port = url.getPort ();
		return port != -1 ? hostname + ":" + port : hostname;
	}
	
	Vector<String> rulebuf = new Vector<String>();
	String[] getRobotExclusionContext(String website, String UserAgent) {
		try {
			URL robotstxtURL = new URL("http://" + website + "/robots.txt");
			URLConnection uc = robotstxtURL.openConnection();
			uc.setRequestProperty("User-Agent", UserAgent);
			uc.connect();
			PushbackInputStream in 
				= new PushbackInputStream(new BufferedInputStream(uc.getInputStream()));
			rulebuf.setSize(0);
			
			boolean relevant = false;
			boolean specific = false;
			String lastFieldName = null;
			while( readLine(in) ) {
				if( fieldName == null ) {
					if( specific )
						break;
					relevant = false;
				}
				else if( fieldName.equals("user-agent") ) {
					if( lastFieldName != null && lastFieldName.equals("disallow") ) {
						if( specific )
							break;
						relevant = false;
					}
					
					if( UserAgent != null && UserAgent.indexOf(fieldValue.toLowerCase()) != -1 ) {
						relevant = true;
						specific = true;
						rulebuf.setSize(0);
					}
					else if( fieldValue.equals("*") ) {
						relevant = true;
						rulebuf.setSize(0);
					}
				}
				else if( relevant && fieldName.equals("disallow") )
					rulebuf.addElement(fieldValue);
				else {
					if( specific )
						break;
					relevant = false;
				}
				lastFieldName = fieldName;
			}
			in.close();
			
			String[] rules = new String[rulebuf.size()];
			rulebuf.copyInto(rules);
			return rules;
		} catch (Exception e) {
			return new String[0];
		}
	}
	
	String fieldName, fieldValue;
    static final int MAX_LINE_LENGTH = 1024;
    StringBuffer linebuf = new StringBuffer();
	boolean readLine(PushbackInputStream in) throws Exception {
		fieldName 	= null;
        fieldValue 	= null;
        linebuf.setLength(0);

        int c;
        int n = 0;
        boolean saw_eoln = false;
        while( true ) {
            c = in.read();
            if( c == -1 )
                break;
            else if( c == '\r' || c == '\n' )
                saw_eoln = true;
            else if( saw_eoln ) {
                in.unread(c);
                break;
            }
            else {
                linebuf.append((char)c);
            }
            ++n;
            if (n == MAX_LINE_LENGTH)
                break;
        }
        if( n == 0 )
            return false;

        String line = linebuf.toString();
        int colon = line.indexOf (':');
        if( colon == -1 ) {
            fieldName 	= null;
            fieldValue 	= null;
        }
        else {
            fieldName  = line.substring(0, colon).trim().toLowerCase();
            fieldValue = line.substring(colon+1).trim();
        }
        return true;
	}
	
	public String getUserAgent() {
		return UserAgent;
	}

	public void setUserAgent(String userAgent) {
		UserAgent = userAgent;
	}
	
	public int getEntryLimit() {
		return cacheEngine.getLimitSize();
	}

	public void setcacheEngine(RobotsTxtCacheEngine<String, String[]> cacheEngine) {
		this.cacheEngine = cacheEngine;
	}
	
}
